{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4b422b46-5dc8-4d97-8fac-c89185382c2a",
   "metadata": {},
   "source": [
    "模型参数的访问、初始化和共享"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e99716fc-65bb-4038-a2c3-b3b0a80dfab5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=4, out_features=3, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=3, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import init\n",
    "net = nn.Sequential(nn.Linear(4, 3), nn.ReLU(), nn.Linear(3, 1)) # pytorch已进⾏默认初始化\n",
    "print(net)\n",
    "X = torch.rand(2, 4)\n",
    "Y = net(X).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb2389c5-71ae-4e40-8349-b732ddb79706",
   "metadata": {},
   "source": [
    "1.1访问多层感知机的所有参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "98b5399c-d306-403c-bed0-6e76a1a23590",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'generator'>\n",
      "0.weight torch.Size([3, 4])\n",
      "0.bias torch.Size([3])\n",
      "2.weight torch.Size([1, 3])\n",
      "2.bias torch.Size([1])\n"
     ]
    }
   ],
   "source": [
    "#通过 Module 类的 parameters() 或者 named_parameters ⽅法来访问所有参数（以迭代器的形式返回）\n",
    "print(type(net.named_parameters()))\n",
    "for name, param in net.named_parameters():\n",
    "    print(name, param.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "33e9b108-afc8-47c8-ba85-90b8b5abb608",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight torch.Size([3, 4]) <class 'torch.nn.parameter.Parameter'>\n",
      "bias torch.Size([3]) <class 'torch.nn.parameter.Parameter'>\n"
     ]
    }
   ],
   "source": [
    "for name, param in net[0].named_parameters():\n",
    "    print(name, param.size(), type(param))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c02a7abb-b8fc-4a1d-ace4-bca346734676",
   "metadata": {},
   "source": [
    "torch.nn.parameter.Parameter是Tensor的子类，如果是这个类型，会自动被添加到模型的参数列表中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "935f597d-0cc0-42d6-89eb-20252f66f039",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weight1\n",
      "weight2\n"
     ]
    }
   ],
   "source": [
    "class MyModel(nn.Module):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(MyModel, self).__init__(**kwargs)\n",
    "        self.weight1 = nn.Parameter(torch.rand(20, 20))  # 需要被训练的参数\n",
    "        self.weight2 = nn.Parameter(torch.rand(20, 20))  # 需要被训练的参数\n",
    "    \n",
    "    def forward(self, x):\n",
    "        pass\n",
    "\n",
    "n = MyModel()\n",
    "for name, param in n.named_parameters():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "07b53e51-a0c5-400b-8529-3a07d06eabc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.4597, -0.1677,  0.1594, -0.1102],\n",
      "        [ 0.3601,  0.3573, -0.2276, -0.1323],\n",
      "        [ 0.4289, -0.1701,  0.4329, -0.3837]])\n",
      "None\n",
      "tensor([[0.2832, 0.4530, 0.4616, 0.4082],\n",
      "        [0.0000, 0.0000, 0.0000, 0.0000],\n",
      "        [0.2300, 0.3680, 0.3749, 0.3316]])\n"
     ]
    }
   ],
   "source": [
    "weight_0 = list(net[0].parameters())[0]\n",
    "print(weight_0.data)\n",
    "print(weight_0.grad) # 反向传播前梯度为None\n",
    "Y.backward()\n",
    "print(weight_0.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "825381d7-cdcd-4a7d-b753-8528a0792197",
   "metadata": {},
   "source": [
    " 1.2初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2438cfac-3de2-4057-b428-f4ad6d29616d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.weight tensor([[-0.0134, -0.0065,  0.0007,  0.0016],\n",
      "        [-0.0070, -0.0071,  0.0019, -0.0200],\n",
      "        [ 0.0091, -0.0023,  0.0195,  0.0026]])\n",
      "2.weight tensor([[-0.0091,  0.0004,  0.0031]])\n"
     ]
    }
   ],
   "source": [
    "for name, param in net.named_parameters():\n",
    "    if 'weight' in name:\n",
    "        #初始化张量的值\n",
    "        init.normal_(param, mean=0, std=0.01)    #均值为0、标准差为0.01的正态分布函数\n",
    "        print(name, param.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "33ace5f4-ef35-4aa0-a1ea-04dcb6090106",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.bias tensor([0., 0., 0.])\n",
      "2.bias tensor([0.])\n"
     ]
    }
   ],
   "source": [
    "for name, param in net.named_parameters():\n",
    "    if 'bias' in name:\n",
    "        init.constant_(param, val=0)\n",
    "        print(name, param.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80a155be-e797-482e-9515-64c1c4ebbb14",
   "metadata": {},
   "source": [
    "1.3自定义初始化方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3433ee25-dd91-41bb-8e13-9584d46b2f02",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normal_(tensor, mean=0, std=1):\n",
    "    with torch.no_grad():\n",
    "        return tensor.normal_(mean, std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12d6fbd5-b0e6-4987-a75f-abda8c666c17",
   "metadata": {},
   "outputs": [],
   "source": [
    "下面这种自定义初始化的方法，令一半概率为0一半概率初始化为均匀分布的随机数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "35d4ad5d-bf5d-4f0c-83ae-5e9a764233e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.weight tensor([[-0.0000, -6.7271, -5.8019, -8.2571],\n",
      "        [-7.8040, -0.0000,  9.3151,  8.1816],\n",
      "        [-0.0000,  0.0000,  8.0369,  7.8595]])\n",
      "2.weight tensor([[-0.0000, 5.3571, 0.0000]])\n"
     ]
    }
   ],
   "source": [
    "#保留绝对值大于5的数，绝对值小于5的变为零\n",
    "def init_weight_(tensor):\n",
    "    with torch.no_grad():\n",
    "        tensor.uniform_(-10, 10)\n",
    "        tensor *= (tensor.abs() >= 5).float()\n",
    "for name, param in net.named_parameters():\n",
    "    if 'weight' in name:\n",
    "        init_weight_(param)\n",
    "        print(name, param.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d76933df-5841-4c3b-a88e-a37d0dcef263",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.bias tensor([2., 2., 2.])\n",
      "2.bias tensor([2.])\n"
     ]
    }
   ],
   "source": [
    "#可以改变模型参数值同时不影响梯度，使得由原本全为0的变成全为1的\n",
    "for name, param in net.named_parameters():\n",
    "    if 'bias' in name:\n",
    "        param.data += 1\n",
    "        print(name, param.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f0cbd01-580e-4965-b86a-3ec2fd4fa43a",
   "metadata": {},
   "source": [
    "1.4共享模型参数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8f6c800-e5e4-479c-a059-39c8ed371d06",
   "metadata": {},
   "source": [
    "Module 类的\n",
    "forward 函数⾥多次调⽤同⼀个层。此外，如果我们传⼊ Sequential 的模块是同⼀个 Module 实例\n",
    "的话参数也是共享的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b9aaa10d-83d3-4ad0-a0cb-61066764320c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=1, out_features=1, bias=False)\n",
      "  (1): Linear(in_features=1, out_features=1, bias=False)\n",
      ")\n",
      "0.weight tensor([[3.]])\n"
     ]
    }
   ],
   "source": [
    "linear = nn.Linear(1, 1, bias=False)\n",
    "net = nn.Sequential(linear, linear) \n",
    "print(net)\n",
    "for name, param in net.named_parameters():\n",
    "    init.constant_(param, val=3)\n",
    "    print(name, param.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05e5d3ac-4a3b-4a37-ae01-19b8eb69c2e3",
   "metadata": {},
   "source": [
    "在内存中两个线形层其实是一个对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e9f47d12-7ec5-4549-9686-a6b408b22d97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "print(id(net[0]) == id(net[1]))\n",
    "print(id(net[0].weight) == id(net[1].weight))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "33216a45-18ff-48ed-9e1d-4b0fe1346d0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(9., grad_fn=<SumBackward0>)\n",
      "tensor([[6.]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.ones(1, 1)\n",
    "y = net(x).sum()\n",
    "print(y)\n",
    "y.backward()\n",
    "print(net[0].weight.grad) # 单次梯度是3，两次所以就是6"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40364127-49be-4bd0-9791-fcedfb41ee47",
   "metadata": {},
   "source": [
    "1.5模型参数的延后初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b52e1fea-362f-48ee-aab9-3454b0e91b6c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch]",
   "language": "python",
   "name": "deep_learning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
